// TODO: Need to break several of these subroutines into their own parts of the ROC package/subroutines.
// TODO: NOTE: ... Many of them are utility, output, etc. that could be organized.
// TODO: NOTE: ... In addition, many are needed to call from the outside.

// TODO: NOTE: I have two ideas for using machine learning to actually learn the ROC curve:
//
//     (i) Use threshold averaging to calculate the empirical curve. Use bootstrap to generate data. Use the convex input NN to learn the function to fit (see the article on laptop).
//     (ii) Bootstrap resample. For each fixed threshold, calculate P(X > lambda) and P(Y > lambda). Generate data according to these probabilities --- calculate the ROC point at threshold (for this bootstrap sample), and calculate numbers of lambda fof this. Feed to RBM.


// NOTE: Bayesian bootstrap would NOT (necessarily) produce a proper ROC curve.
//
// NOTE: ... See Figs. 3 and 4 in J. Gu, et al. (2008).


// NOTE: There are (at least) two ways to calculate the ROC curve:
//
//     (i) uniform in FPR
//     (ii) uniform in TPR
//
// NOTE: ... The former seems preferable for actual use (uniform when plotting, etc.). This is used throughout, as standard.


// NOTE: An approximation made in several subroutines is that the ROC curve can be constructed as piecewise linear.
//
// NOTE: ... Since they (TPR vs FPR) can (often) be calculated arbitrarily dense, this is a convergent approximation.


#ifndef STATSxx_POSTPROCESS_ROC_HPP
#define STATSxx_POSTPROCESS_ROC_HPP


// STL
#include <string>  // std::string
#include <tuple>   // std::tuple<>
#include <utility> // std::pair<>
#include <vector>  // std::vector<>

// stats++
#include "statsxx/postprocess/ROC/ROC_pt.hpp"         // ROC_pt
#include "statsxx/postprocess/ROC/ROC_CostMatrix.hpp" // ROC_CostMatrix


//=========================================================
// EMPIRICAL ROC CURVE
//=========================================================

std::vector<ROC_pt> calc_ROC_curve(
                                   const std::vector<int>    &L,
                                   const std::vector<double> &f
                                   );
#include "statsxx/postprocess/ROC/calc_ROC_curve.cpp"


std::vector<std::vector<ROC_pt>> divide_and_calc_ROCs(
                                                      const std::vector<int>    &L,
                                                      const std::vector<double> &f,
                                                      // -----
                                                      const int                  nROCs
                                                      );
#include "statsxx/postprocess/ROC/divide_and_calc_ROCs.cpp"

ROC_pt ROC_pt_at_thresh(
                        const std::vector<ROC_pt> &ROC,
                        const double               thresh
                        );
#include "statsxx/postprocess/ROC/ROC_pt_at_thresh.cpp"

std::pair<
          std::vector<ROC_pt>, // ROC
          std::vector<ROC_pt>  // stddevROC
          > calc_ROC_curve_thresh_avg(
                                      const std::vector<int>                    &L,
                                      const std::vector<double>                 &f,
                                      // -----
                                      const int                                  nthresh,
                                      const int                                  nROCs
                                      );
#include "statsxx/postprocess/ROC/calc_ROC_curve_thresh_avg.cpp"


//=========================================================
// ESTIMATE (FIT) ROC CURVE
//=========================================================

std::tuple<
           bool,
           std::string,
           double,
           double
           > fit_binormal(
                          std::vector<ROC_pt> ROC,
                          std::vector<ROC_pt> stddevROC,
                          // -----
                          const int           nROCs
                          );
#include "statsxx/postprocess/ROC/fit_binormal.cpp"


std::tuple<
           bool,
           std::string,
           double,
           double
           > fit_binormal_LR(
                             std::vector<ROC_pt> ROC,
                             std::vector<ROC_pt> stddevROC,
                             // -----
                             const int           nROCs
                             );
#include "statsxx/postprocess/ROC/fit_binormal_LR.cpp"


std::vector<ROC_pt> binormal_to_ROC(
                                    const double        a,
                                    const double        b,
                                    // -----
                                    const unsigned int  n
                                    );
#include "statsxx/postprocess/ROC/binormal_to_ROC.cpp"


std::vector<ROC_pt> binormal_LR_to_ROC(
                                       const double        d_a, // Metz and Pan binormal-LR parameter
                                       const double        c,   // Metz and Pan binormal-LR parameter
                                       // -----
                                       const unsigned int  n
                                       );
#include "statsxx/postprocess/ROC/binormal_LR_to_ROC.cpp"


std::vector<ROC_pt> map_scores(
                               const std::vector<ROC_pt> &ROC_scores,
                               // -----
                               std::vector<ROC_pt>        ROC
                               );
#include "statsxx/postprocess/ROC/map_scores.cpp"

//=========================================================
// CALCULATE AREA UNDER (ROC) CURVES (AUC)
//=========================================================

// NOTE: (Specialized) subroutines to calculate the AUC for models, where possible.

double AUC_binormal(
                    const double a,
                    const double b
                    );
#include "statsxx/postprocess/ROC/AUC_binormal.cpp"


double AUC_binormal_LR(
                       const double d_a,
                       const double c
                       );
#include "statsxx/postprocess/ROC/AUC_binormal_LR.cpp"


double AUC_numerical(
                     const std::vector<ROC_pt> &ROC
                     );
#include "statsxx/postprocess/ROC/AUC_numerical.cpp"

//=========================================================
// CUTOFFS
//=========================================================

std::pair<
          bool,
          double
          > cutoff_CB_well_behaved(
                                   const std::vector<ROC_pt> &ROC,
                                   const std::vector<double> &slopes,
                                   // -----
                                   const double               m
                                   );
#include "statsxx/postprocess/ROC/cutoff_CB_well_behaved.cpp"

std::pair<
          bool,
          double
          > cutoff_CB(
                      const std::vector<ROC_pt> &ROC,
                      // -----
                      const ROC_CostMatrix      &cm
                      );
#include "statsxx/postprocess/ROC/cutoff_CB.cpp"

// TODO: NOTE: May be able to add a specialized subroutine for determining cutoffs from estimated (fit) ROC curves.
//
// TODO: NOTE: ... I did a bit of searching for a binormal curve, but could not figure out how to do this (easily, at least).
// TODO: NOTE: ... But I am sure that it should be possible.

//=========================================================
// INPUT/OUTPUT SUBROUTINES
//=========================================================

std::pair<
          std::vector<ROC_pt>, // ROC
          std::vector<ROC_pt>  // stddevROC
          > read_ROC_curve(
                           const std::string filename
                           );
#include "statsxx/postprocess/ROC/read_ROC_curve.cpp"


void output_ROC_curve(
                      const std::vector<ROC_pt> &ROC,
                      // -----
                      const std::string          filename
                      );

void output_ROC_curve(
                      const std::vector<ROC_pt> &ROC,
                      const std::vector<ROC_pt> &stddevROC,
                      // -----
                      const std::string          filename
                      );
#include "statsxx/postprocess/ROC/output_ROC_curve.cpp"


void output_AUC(
                const double      _AUC,
                // -----
                const std::string filename
                );
#include "statsxx/postprocess/ROC/output_AUC.cpp"


void output_cutoff(
                   const double      _cutoff,
                   // -----
                   const std::string filename
                   );
#include "statsxx/postprocess/ROC/output_cutoff.cpp"


/*
// *****

namespace statsxx
{
    namespace ROC
    {
        double                  opt_cutoff_maxSumSeSp( const std::vector<ROC_pt> &ROC );
        std::pair<bool, double> opt_cutoff_CB( const std::vector<ROC_pt> &ROC, const ROC_CostMatrix &cm );
    }
}
*/

#endif
